def reciprocal_rank_fusion(results_lists, k=60):
    """Combine multiple ranked lists using RRF."""
    fused_scores = {}
    doc_map = {}

    for result_list in results_lists:
        for rank, doc in enumerate(result_list):
            doc_id = doc['id']
            if doc_id not in doc_map:
                doc_map[doc_id] = doc
            prev = fused_scores.get(doc_id, 0.0)
            fused_scores[doc_id] = prev + 1/(rank + k)

    # Sort by fused score (descending)
    ranked = sorted(fused_scores.items(),
                    key=lambda x: x[1], reverse=True)
    return [doc_map[doc_id] for doc_id, _ in ranked]

class AdvancedRetriever:
    def __init__(self, vector_store,
                 reranker_model="BAAI/bge-reranker-base"):
        self.vector_store = vector_store
        self.reranker = CrossEncoder(reranker_model)

    def rerank(self, query, candidates, top_k=5):
        """Re-rank candidates using Cross-Encoder."""
        pairs = [[query, doc['content']]
                 for doc in candidates]

        scores = self.reranker.predict(pairs)
        for i, doc in enumerate(candidates):
            doc['relevance_score'] = float(scores[i])

        ranked = sorted(candidates,
                       key=lambda x: x['relevance_score'],
                       reverse=True)
        return ranked[:top_k]

def retrieve(self, query, top_k=5, candidates=50):
    """Full pipeline: Hybrid → RRF → Re-rank → Top K"""

    # 1. Semantic search (dense)
    semantic = self._retrieve_semantic(query, candidates)

    # 2. Keyword search (sparse)
    keyword = self._retrieve_keyword(query, candidates)

    # 3. Fuse with RRF
    if keyword:
        hybrid = reciprocal_rank_fusion(
            [semantic, keyword])
    else:
        hybrid = semantic

    # 4. Re-rank top candidates
    return self.rerank(query, hybrid[:candidates], top_k)

class QueryPreprocessor:
    """Cleans and normalizes queries before retrieval."""

    def __init__(self, acronym_map=None):
        self.acronym_map = acronym_map or {
            "RAG": "Retrieval-Augmented Generation",
            "LLM": "Large Language Model",
            "NLP": "Natural Language Processing",
        }

    def preprocess(self, query: str) -> str:
        query = re.sub(r'\s+', ' ', query.strip())
        for acronym, expansion in self.acronym_map.items():
            if acronym in query.upper():
                query = f"{query} ({expansion})"
        return query

def classify_query_intent(query: str) -> str:
    """Simple rule-based intent classifier."""
    q = query.lower()

    if any(w in q for w in ["compare", "vs", "difference"]):
        return "comparison"
    elif any(w in q for w in ["how to", "implement", "build"]):
        return "how_to"
    elif any(w in q for w in ["why", "error", "fail"]):
        return "troubleshooting"
    else:
        return "factual"

def generate_multi_queries(original_query, n=3):
    """Generate query variations using an LLM."""
    prompt = f"""Generate {n} different search queries
    that would help answer: "{original_query}"
    from a database of technical research papers.
    Provide only the queries, one per line."""

    response = llm.complete(prompt)
    queries = [original_query]  # Always include original
    queries.extend(response.strip().split('\n'))
    return queries[:n + 1]

class QueryCache:
    """In-memory cache for repeated query patterns."""
    def __init__(self, max_size=1000):
        self._cache = {}

    def _hash_query(self, query):
        normalized = query.lower().strip()
        return hashlib.md5(normalized.encode()).hexdigest()

    def get(self, query):
        return self._cache.get(self._hash_query(query))

    def set(self, query, results):
        if len(self._cache) >= self._max_size:
            oldest = next(iter(self._cache))
            del self._cache[oldest]
        self._cache[self._hash_query(query)] = results